package com.wfg.bigdata.spark.sql

import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}

object Spark01_SparkSQL_Basic_file {

    def main(args: Array[String]): Unit = {

        // TODO 创建SparkSQL的运行环境
        val sparkConf = new SparkConf().setMaster("local[*]").setAppName("sparkSQL")
        val spark = SparkSession.builder().config(sparkConf).getOrCreate()
        // 在使用DataFrame时，如果涉及到转换操作，需要引入转换规则
        import spark.implicits._


        // TODO 执行逻辑操作
        spark.udf.register("prefixName", (name:String) => {
            val strings = name.split("-")
            strings.length
        })
        // TODO DataFrame
        val frame = spark.read.option("encoding","GBK").option("header","true").csv("datas/sql/product.csv")
        frame.createTempView("product");
        val frame1 = spark.sql("select prefixName(`全路径名称`) from product ")

        frame1.write.mode("append").csv("output4")
        // TODO 关闭环境
        spark.close()
    }

}
